Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.172-b11 mixed mode): "Attach Listener" #177 daemon prio=9 os_prio=0 tid=0x00007f482c010800 nid=0x1f8 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "ReplicaFetcherThread-2-9" #170 prio=5 os_prio=0 tid=0x00007f37fc013800 nid=0x1d9 runnable [0x00007f38190dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000659ec28b0> (a sun.nio.ch.Util$3) - locked <0x0000000659ec28a0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000659ec0708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-9" #169 prio=5 os_prio=0 tid=0x00007f37fc012800 nid=0x1d8 runnable [0x00007f38188d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000659dfc298> (a sun.nio.ch.Util$3) - locked <0x0000000659dfc288> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000659dfc2a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-9" #168 prio=5 os_prio=0 tid=0x00007f37fc01e800 nid=0x1d7 runnable [0x00007f38191dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000659ede410> (a sun.nio.ch.Util$3) - locked <0x0000000659ede400> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000659edc258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-9" #167 prio=5 os_prio=0 tid=0x00007f37fc01e000 nid=0x1d6 runnable [0x00007f3818fdb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000659dfc640> (a sun.nio.ch.Util$3) - locked <0x0000000659dfc630> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000659dfc650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "RMI TCP Connection(134)-172.30.4.72" #163 daemon prio=5 os_prio=0 tid=0x00007f481400e800 nid=0x1d2 runnable [0x00007f38194df000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007ad130710> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(FilterInputStream.java:83) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:555) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:835) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:688) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$781/9871554.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:687) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x0000000659947bb0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "executor-Heartbeat" #146 prio=5 os_prio=0 tid=0x00007f37f8036000 nid=0x1c1 waiting on condition [0x00007f3819fe9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c45ce28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-10" #143 prio=5 os_prio=0 tid=0x00007f47b8057800 nid=0x1be runnable [0x00007f38189d5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000656eb4a48> (a sun.nio.ch.Util$3) - locked <0x0000000656eb4a38> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000656eb4a58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-10" #142 prio=5 os_prio=0 tid=0x00007f47b8071000 nid=0x1bd runnable [0x00007f38192de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000656ecbde0> (a sun.nio.ch.Util$3) - locked <0x0000000656ecbdd0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000656ecbdf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-10" #141 prio=5 os_prio=0 tid=0x00007f47b8070800 nid=0x1bc runnable [0x00007f37caf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000656ee3178> (a sun.nio.ch.Util$3) - locked <0x0000000656ee3168> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000656ee3188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-10" #140 prio=5 os_prio=0 tid=0x00007f47b806f800 nid=0x1bb runnable [0x00007f38186d2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000656ecc188> (a sun.nio.ch.Util$3) - locked <0x0000000656ecc178> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000656ecc198> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "RMI TCP Connection(132)-172.30.4.72" #135 daemon prio=5 os_prio=0 tid=0x00007f4814003800 nid=0x1b6 runnable [0x00007f3819be6000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000007ad2f6150> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(FilterInputStream.java:83) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:555) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:835) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:688) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$781/9871554.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:687) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000006569cdf90> (a java.util.concurrent.ThreadPoolExecutor$Worker) "executor-Fetch" #97 prio=5 os_prio=0 tid=0x00007f381000c800 nid=0x190 waiting on condition [0x00007f38181cf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c3f7700> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-8" #96 prio=5 os_prio=0 tid=0x00007f47cc07b000 nid=0x18f runnable [0x00007f38184d0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653551c28> (a sun.nio.ch.Util$3) - locked <0x0000000653551c18> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653551bd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-6" #95 prio=5 os_prio=0 tid=0x00007f47cc079000 nid=0x18e runnable [0x00007f38185d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653568880> (a sun.nio.ch.Util$3) - locked <0x0000000653568870> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653568828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-6" #93 prio=5 os_prio=0 tid=0x00007f47cc075000 nid=0x18c runnable [0x00007f38187d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653551fd0> (a sun.nio.ch.Util$3) - locked <0x0000000653551fc0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653551f78> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-8" #90 prio=5 os_prio=0 tid=0x00007f47cc048000 nid=0x189 runnable [0x00007f3818ad6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653552720> (a sun.nio.ch.Util$3) - locked <0x0000000653552710> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006535526c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-6" #89 prio=5 os_prio=0 tid=0x00007f47cc046000 nid=0x188 runnable [0x00007f3818bd7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653568c28> (a sun.nio.ch.Util$3) - locked <0x0000000653568c18> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653568bd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-8" #88 prio=5 os_prio=0 tid=0x00007f47cc044000 nid=0x187 runnable [0x00007f3818cd8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653552ac8> (a sun.nio.ch.Util$3) - locked <0x0000000653552ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653552a70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-0" #87 prio=5 os_prio=0 tid=0x00007f47cc042800 nid=0x186 runnable [0x00007f3818dd9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006535af278> (a sun.nio.ch.Util$3) - locked <0x00000006535af268> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006535af220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-7" #86 prio=5 os_prio=0 tid=0x00007f47cc040800 nid=0x185 runnable [0x00007f3818eda000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653552e70> (a sun.nio.ch.Util$3) - locked <0x0000000653552e60> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653552e18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-7" #81 prio=5 os_prio=0 tid=0x00007f47cc0b6800 nid=0x180 runnable [0x00007f38193df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006535535c0> (a sun.nio.ch.Util$3) - locked <0x00000006535535b0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653553568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-8" #79 prio=5 os_prio=0 tid=0x00007f47cc0b2800 nid=0x17e runnable [0x00007f38195e1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006535af9c8> (a sun.nio.ch.Util$3) - locked <0x00000006535af9b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006535af970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-7" #78 prio=5 os_prio=0 tid=0x00007f47cc0c4000 nid=0x17d runnable [0x00007f38196e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653553968> (a sun.nio.ch.Util$3) - locked <0x0000000653553958> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653553910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-7" #77 prio=5 os_prio=0 tid=0x00007f47cc0c5800 nid=0x17c runnable [0x00007f38197e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000653569988> (a sun.nio.ch.Util$3) - locked <0x0000000653ca5d50> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000653ca5b58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-6" #76 prio=5 os_prio=0 tid=0x00007f47cc0be800 nid=0x17b runnable [0x00007f38198e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006535afd70> (a sun.nio.ch.Util$3) - locked <0x00000006535afd60> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006535afd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:107) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:197) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:286) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:133) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread$$Lambda$874/1026273165.apply(Unknown Source) at scala.Option.foreach(Option.scala:274) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:132) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "RMI TCP Connection(idle)" #74 daemon prio=5 os_prio=0 tid=0x00007f4814006800 nid=0x179 waiting on condition [0x00007f3819ae6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649dd8700> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Connection(72)-172.30.4.72" #73 daemon prio=5 os_prio=0 tid=0x00007f4814005000 nid=0x178 runnable [0x00007f38199e4000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x00000006565bbab0> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(FilterInputStream.java:83) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:555) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:835) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:688) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$781/9871554.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:687) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x0000000653413018> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI Scheduler(0)" #70 daemon prio=5 os_prio=0 tid=0x00007f47e40ea800 nid=0x175 waiting on condition [0x00007f3819ce8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649dd5558> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-network-thread-11-ListenerName(PLAINTEXT)-PLAINTEXT-2" #68 prio=5 os_prio=0 tid=0x00007f48b9717000 nid=0x173 runnable [0x00007f381a2ea000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000064c2b6808> (a sun.nio.ch.Util$3) - locked <0x000000064c2b67f8> (a java.util.Collections$UnmodifiableSet) - locked <0x000000064c2b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at kafka.network.Processor.poll(SocketServer.scala:863) at kafka.network.Processor.run(SocketServer.scala:762) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-network-thread-11-ListenerName(PLAINTEXT)-PLAINTEXT-1" #67 prio=5 os_prio=0 tid=0x00007f48b9715800 nid=0x172 runnable [0x00007f381a3eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000064c2b6b30> (a sun.nio.ch.Util$3) - locked <0x000000064c2b6b20> (a java.util.Collections$UnmodifiableSet) - locked <0x000000064c2b6b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at kafka.network.Processor.poll(SocketServer.scala:863) at kafka.network.Processor.run(SocketServer.scala:762) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-network-thread-11-ListenerName(PLAINTEXT)-PLAINTEXT-0" #66 prio=5 os_prio=0 tid=0x00007f48b9714800 nid=0x171 runnable [0x00007f381a4ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000064c2b6e58> (a sun.nio.ch.Util$3) - locked <0x000000064c2b6e48> (a java.util.Collections$UnmodifiableSet) - locked <0x000000064c2b6e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at kafka.network.Processor.poll(SocketServer.scala:863) at kafka.network.Processor.run(SocketServer.scala:762) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #65 prio=5 os_prio=0 tid=0x00007f48b970a000 nid=0x170 waiting on condition [0x00007f381a5ed000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b7140> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-11" #64 daemon prio=5 os_prio=0 tid=0x00007f48b96f7000 nid=0x16f waiting on condition [0x00007f381a6ee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-10" #63 daemon prio=5 os_prio=0 tid=0x00007f48b96f5000 nid=0x16e waiting on condition [0x00007f381a7ef000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-9" #62 daemon prio=5 os_prio=0 tid=0x00007f48b96f3800 nid=0x16d waiting on condition [0x00007f381a8f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-8" #61 daemon prio=5 os_prio=0 tid=0x00007f48b96f1800 nid=0x16c waiting on condition [0x00007f381a9f1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-7" #60 daemon prio=5 os_prio=0 tid=0x00007f48b96f0000 nid=0x16b waiting on condition [0x00007f381aaf2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-6" #59 daemon prio=5 os_prio=0 tid=0x00007f48b96ee000 nid=0x16a waiting on condition [0x00007f381abf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-5" #58 daemon prio=5 os_prio=0 tid=0x00007f48b96ec800 nid=0x169 waiting on condition [0x00007f381acf4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-4" #57 daemon prio=5 os_prio=0 tid=0x00007f48b96ea800 nid=0x168 waiting on condition [0x00007f381adf5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-3" #56 daemon prio=5 os_prio=0 tid=0x00007f48b96e9000 nid=0x167 waiting on condition [0x00007f381aef6000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-2" #55 daemon prio=5 os_prio=0 tid=0x00007f48b96e7800 nid=0x166 waiting on condition [0x00007f381aff7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-1" #54 daemon prio=5 os_prio=0 tid=0x00007f48b96e4000 nid=0x165 waiting on condition [0x00007f381b0f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "data-plane-kafka-request-handler-0" #53 daemon prio=5 os_prio=0 tid=0x00007f48b96e3000 nid=0x164 waiting on condition [0x00007f381b1f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b78a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:344) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-11" #51 prio=5 os_prio=0 tid=0x00007f48b8c63000 nid=0x163 runnable [0x00007f381b2fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000064c2b8cc0> (a sun.nio.ch.Util$3) - locked <0x000000064c2b8cb0> (a java.util.Collections$UnmodifiableSet) - locked <0x000000064c2b8c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:794) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:539) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "transaction-log-manager-0" #52 daemon prio=5 os_prio=0 tid=0x00007f48b8c61000 nid=0x162 waiting on condition [0x00007f381b3fb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b8f38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #49 daemon prio=5 os_prio=0 tid=0x00007f48b8c2c000 nid=0x161 waiting on condition [0x00007f381b6fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b91a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-11-Rebalance" #48 prio=5 os_prio=0 tid=0x00007f48b8c18000 nid=0x160 waiting on condition [0x00007f381b7fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b9420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:433) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:459) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ExpirationReaper-11-Heartbeat" #47 prio=5 os_prio=0 tid=0x00007f48b8c17000 nid=0x15f runnable [0x00007f381b8fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b9668> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:433) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:459) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ExpirationReaper-11-topic" #46 prio=5 os_prio=0 tid=0x00007f48b8c07800 nid=0x15e waiting on condition [0x00007f381b9ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b98b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:433) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:459) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "controller-event-thread" #45 prio=5 os_prio=0 tid=0x00007f48b8c06000 nid=0x15d waiting on condition [0x00007f381bb00000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b9af8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:127) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "LogDirFailureHandler" #44 prio=5 os_prio=0 tid=0x00007f48b8a52000 nid=0x15c waiting on condition [0x00007f47ac1fc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064906bb80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:56) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:205) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ExpirationReaper-11-ElectPreferredLeader" #43 prio=5 os_prio=0 tid=0x00007f48b8a3a800 nid=0x15b runnable [0x00007f47ac2fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2b9ed0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:433) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:459) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ExpirationReaper-11-DeleteRecords" #42 prio=5 os_prio=0 tid=0x00007f48b8a39800 nid=0x15a runnable [0x00007f47ac3fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2ba118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:433) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:459) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ExpirationReaper-11-Fetch" #41 prio=5 os_prio=0 tid=0x00007f48b8a35000 nid=0x159 runnable [0x00007f47b41fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2ba360> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:433) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:459) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ExpirationReaper-11-Produce" #40 prio=5 os_prio=0 tid=0x00007f48b8a33800 nid=0x158 runnable [0x00007f47c43fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2ba5a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:433) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:459) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-9092" #39 prio=5 os_prio=0 tid=0x00007f48b9501000 nid=0x157 runnable [0x00007f47d41fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000064c2ba878> (a sun.nio.ch.Util$3) - locked <0x000000064c2ba868> (a java.util.Collections$UnmodifiableSet) - locked <0x000000064c2ba820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:534) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #38 prio=5 os_prio=0 tid=0x00007f48b8936000 nid=0x156 waiting on condition [0x00007f47f3bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064a4cb050> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:75) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:309) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "kafka-scheduler-9" #37 daemon prio=5 os_prio=0 tid=0x00007f47e407e800 nid=0x155 waiting on condition [0x00007f47d42fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #36 daemon prio=5 os_prio=0 tid=0x00007f47e407d000 nid=0x154 waiting on condition [0x00007f47d43fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #35 daemon prio=5 os_prio=0 tid=0x00007f47e407b000 nid=0x153 waiting on condition [0x00007f47f01f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #34 daemon prio=5 os_prio=0 tid=0x00007f47e4079000 nid=0x152 waiting on condition [0x00007f47f02f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #33 daemon prio=5 os_prio=0 tid=0x00007f47e4077800 nid=0x151 waiting on condition [0x00007f47f03f6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-4" #32 daemon prio=5 os_prio=0 tid=0x00007f47e4075800 nid=0x150 waiting on condition [0x00007f47f04f7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #31 daemon prio=5 os_prio=0 tid=0x00007f47e4074000 nid=0x14f waiting on condition [0x00007f47f05f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #30 daemon prio=5 os_prio=0 tid=0x00007f47e4071000 nid=0x14e waiting on condition [0x00007f47f06f9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #29 daemon prio=5 os_prio=0 tid=0x00007f47e4070000 nid=0x14d waiting on condition [0x00007f47f11fa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #28 daemon prio=5 os_prio=0 tid=0x00007f47e4061800 nid=0x14c waiting on condition [0x00007f47f26fb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649531d88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #26 prio=5 os_prio=0 tid=0x00007f48ba711000 nid=0x14a waiting on condition [0x00007f47f3efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000649169050> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #25 prio=5 os_prio=0 tid=0x00007f48ba70f800 nid=0x149 waiting on condition [0x00007f47f3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064906bd90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #24 prio=5 os_prio=0 tid=0x00007f48ba70d000 nid=0x148 waiting on condition [0x00007f4818100000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064905f2e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:89) Locked ownable synchronizers: - None "SensorExpiryThread" #23 daemon prio=5 os_prio=0 tid=0x00007f48ba6f9800 nid=0x147 waiting on condition [0x00007f4818c20000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064906bdd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #22 daemon prio=5 os_prio=0 tid=0x00007f48ba5c9000 nid=0x146 waiting on condition [0x00007f4818601000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064905f4d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(172.30.4.132:2180)" #21 daemon prio=5 os_prio=0 tid=0x00007f48ba56f000 nid=0x145 runnable [0x00007f4818b1f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000006490753b0> (a sun.nio.ch.Util$3) - locked <0x00000006490753a0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000649075358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #18 daemon prio=5 os_prio=0 tid=0x00007f48ba536800 nid=0x142 waiting on condition [0x00007f4818f21000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064905e4b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #17 daemon prio=5 os_prio=0 tid=0x00007f48ba534000 nid=0x141 waiting on condition [0x00007f4819022000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064905e4b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #14 daemon prio=5 os_prio=0 tid=0x00007f48b897c000 nid=0x13f runnable [0x00007f4819824000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-9999" #13 daemon prio=5 os_prio=0 tid=0x00007f48b8962800 nid=0x13e runnable [0x00007f4819925000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #12 daemon prio=5 os_prio=0 tid=0x00007f48b894f800 nid=0x13d runnable [0x00007f484c14f000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #10 daemon prio=9 os_prio=0 tid=0x00007f48b84f5800 nid=0x13c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f48b84e2800 nid=0x13b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f48b84e0800 nid=0x13a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f48b84de000 nid=0x139 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f48b84dc000 nid=0x138 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f48b84da000 nid=0x137 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f48b84d8800 nid=0x136 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f48b84a6000 nid=0x135 in Object.wait() [0x00007f484d01f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x000000064906aae0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f48b84a3800 nid=0x134 in Object.wait() [0x00007f484d120000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x000000064906aad0> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f48b800e800 nid=0x11e waiting on condition [0x00007f48bfda8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000064c2cdb38> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:670) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:85) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f48b849a000 nid=0x133 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b8025800 nid=0x11f runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b8027000 nid=0x120 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b8029000 nid=0x121 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b802a800 nid=0x122 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b802c800 nid=0x123 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b802e000 nid=0x124 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b8030000 nid=0x125 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f48b8031800 nid=0x126 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f48b804a000 nid=0x130 runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f48b804c000 nid=0x131 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f48b804d800 nid=0x132 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f48b8042800 nid=0x12f runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f48b8040800 nid=0x12e runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f48b803f000 nid=0x12d runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f48b803d000 nid=0x12c runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f48b803b800 nid=0x12b runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f48b8039800 nid=0x12a runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f48b8038000 nid=0x129 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f48b8036000 nid=0x128 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f48b8034800 nid=0x127 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f48b897e800 nid=0x140 waiting on condition JNI global references: 2988