2016-11-09 22:16:34 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.72-b15 mixed mode): "Attach Listener" #4407 daemon prio=9 os_prio=0 tid=0x00007f698005d800 nid=0xeb32f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RMI TCP Connection(idle)" #4406 daemon prio=5 os_prio=0 tid=0x00007f692005f800 nid=0xeadfe waiting on condition [0x00007f5aedefc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8ab66d8> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "RMI TCP Connection(17796)-2a02:6b8:b010:7801:0:0:0:4" #4405 daemon prio=5 os_prio=0 tid=0x00007f6920067000 nid=0xea822 runnable [0x00007f5cdb9f9000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.$$YJP$$socketRead0(Native Method) at java.net.SocketInputStream.socketRead0(SocketInputStream.java) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:170) at java.net.SocketInputStream.read(SocketInputStream.java:141) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x000000078dc02050> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(FilterInputStream.java:83) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$2/559123700.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "JMX server connection timeout 4351" #4351 daemon prio=5 os_prio=0 tid=0x00007f1c6c1cc800 nid=0xb6ee5 in Object.wait() [0x00007f3f753d0000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.$$YJP$$wait(Native Method) at java.lang.Object.wait(Object.java) at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:168) - locked <0x0000000448138258> (a [I) at java.lang.Thread.run(Thread.java:745) "ReplicaFetcherThread-1-8" #3425 prio=5 os_prio=0 tid=0x00007f5ce80da000 nid=0x93925 runnable [0x00007f64665fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003f3eb0b10> (a sun.nio.ch.Util$2) - locked <0x00000003f3eb0b00> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003f3eb0ab8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-8" #3424 prio=5 os_prio=0 tid=0x00007f5ce80c0000 nid=0x93924 runnable [0x00007f5b2d5fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000041c13c2a0> (a sun.nio.ch.Util$2) - locked <0x000000041c13c290> (a java.util.Collections$UnmodifiableSet) - locked <0x000000041c13c248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-6" #2464 prio=5 os_prio=0 tid=0x00007f5ce818b000 nid=0x8ee42 runnable [0x00007f5b2c9ef000] java.lang.Thread.State: RUNNABLE at scala.Tuple2.equals(Tuple2.scala:20) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:940) at kafka.utils.Pool.get(Pool.scala:69) at kafka.server.ReplicaManager.getPartition(ReplicaManager.scala:286) at kafka.server.ReplicaManager.getReplica(ReplicaManager.scala:317) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:126) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:159) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:141) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:141) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:138) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:136) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-6" #2463 prio=5 os_prio=0 tid=0x00007f5ce80fd000 nid=0x8ee41 runnable [0x00007f5cdb7f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.$$YJP$$read0(Native Method) at sun.nio.ch.FileDispatcherImpl.read0(FileDispatcherImpl.java) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x0000000417bbf970> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:110) at org.apache.kafka.common.network.NetworkReceive.readFromReadableChannel(NetworkReceive.java:97) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:71) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:154) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:135) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:343) at org.apache.kafka.common.network.Selector.poll(Selector.java:291) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-7" #1791 prio=5 os_prio=0 tid=0x00007f5ce8153000 nid=0xe390a runnable [0x00007f5b2d0f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.$$YJP$$current(Native Method) at sun.nio.ch.NativeThread.current(NativeThread.java) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00000003f58bd8f0> (a java.lang.Object) - locked <0x00000003f58bd8d0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:110) at org.apache.kafka.common.network.NetworkReceive.readFromReadableChannel(NetworkReceive.java:81) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:71) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:154) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:135) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:343) at org.apache.kafka.common.network.Selector.poll(Selector.java:291) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-7" #1790 prio=5 os_prio=0 tid=0x00007f5ce8139000 nid=0xe3909 runnable [0x00007f5cd84ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004271f7a70> (a sun.nio.ch.Util$2) - locked <0x00000004271f7a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004271f7a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-15" #1789 prio=5 os_prio=0 tid=0x00007f5d1410a800 nid=0xdaa43 runnable [0x00007f3f751cd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.$$YJP$$current(Native Method) at sun.nio.ch.NativeThread.current(NativeThread.java) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00000004273d1950> (a java.lang.Object) - locked <0x00000004273d1930> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:110) at org.apache.kafka.common.network.NetworkReceive.readFromReadableChannel(NetworkReceive.java:81) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:71) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:154) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:135) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:343) at org.apache.kafka.common.network.Selector.poll(Selector.java:291) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-15" #1788 prio=5 os_prio=0 tid=0x00007f5d140c8800 nid=0xdaa42 runnable [0x00007f5aed9f6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:71) at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:22) at org.apache.kafka.common.protocol.types.ArrayOf.read(ArrayOf.java:73) at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:71) at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:22) at org.apache.kafka.common.protocol.types.ArrayOf.read(ArrayOf.java:73) at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:71) at org.apache.kafka.clients.NetworkClient.parseResponse(NetworkClient.java:380) at org.apache.kafka.clients.NetworkClient.handleCompletedReceives(NetworkClient.java:449) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:269) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-20" #556 prio=5 os_prio=0 tid=0x00007f5d0c0ca800 nid=0x3324a runnable [0x00007f59fdb82000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003dd6e7ba0> (a sun.nio.ch.Util$2) - locked <0x00000003dd6e7b90> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003dd6e7b48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-20" #555 prio=5 os_prio=0 tid=0x00007f5d0c096000 nid=0x33249 runnable [0x00007f5aedbf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003dd6a8420> (a sun.nio.ch.Util$2) - locked <0x00000003dd6a8410> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003dd6a83c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-12" #552 prio=5 os_prio=0 tid=0x00007f5ce00c4000 nid=0x30fa8 runnable [0x00007f5b2d1f8000] java.lang.Thread.State: RUNNABLE at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.map(TraversableLike.scala:245) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$1.apply(AbstractFetcherThread.scala:93) at kafka.server.AbstractFetcherThread$$anonfun$1.apply(AbstractFetcherThread.scala:92) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:92) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-12" #551 prio=5 os_prio=0 tid=0x00007f5ce0090000 nid=0x30fa7 runnable [0x00007f5b2c5eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.$$YJP$$read0(Native Method) at sun.nio.ch.FileDispatcherImpl.read0(FileDispatcherImpl.java) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00000003f17636c8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:110) at org.apache.kafka.common.network.NetworkReceive.readFromReadableChannel(NetworkReceive.java:97) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:71) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:154) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:135) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:343) at org.apache.kafka.common.network.Selector.poll(Selector.java:291) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-19" #519 prio=5 os_prio=0 tid=0x00007f5cd008d800 nid=0x88c4 runnable [0x00007f5b2c8ef000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:946) at kafka.utils.Pool.get(Pool.scala:69) at kafka.cluster.Partition.getReplica(Partition.scala:111) at kafka.server.ReplicaManager.getReplica(ReplicaManager.scala:320) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:126) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:159) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:141) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:141) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:138) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:136) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-19" #518 prio=5 os_prio=0 tid=0x00007f5cd008b800 nid=0x88c3 runnable [0x00007f5aedffc000] java.lang.Thread.State: RUNNABLE at kafka.log.Log.append(Log.scala:340) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:140) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:159) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:141) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:141) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:138) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:136) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-18" #515 prio=5 os_prio=0 tid=0x00007f5d240db800 nid=0x2406 runnable [0x00007f5b2ccf3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004270c3390> (a sun.nio.ch.Util$2) - locked <0x00000004270c3380> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004270c3338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-18" #514 prio=5 os_prio=0 tid=0x00007f5d2405c000 nid=0x2405 runnable [0x00007f5b2caf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003f3eb0c48> (a sun.nio.ch.Util$2) - locked <0x00000003f3eb0c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003f3eb0bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-17" #511 prio=5 os_prio=0 tid=0x00007f5cf407d800 nid=0xff576 runnable [0x00007f5aeddfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000427329b60> (a sun.nio.ch.Util$2) - locked <0x0000000427329b50> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000427329b08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-17" #510 prio=5 os_prio=0 tid=0x00007f5cf408a000 nid=0xff575 runnable [0x00007f5b2cef4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003f3c000f0> (a sun.nio.ch.Util$2) - locked <0x00000003f3c000e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003f3c00098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-16" #495 prio=5 os_prio=0 tid=0x00007f5d28108000 nid=0xf4cb7 runnable [0x00007f5cd85ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003fd4cfaa0> (a sun.nio.ch.Util$2) - locked <0x00000003fd4cfa90> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003fd4cfa38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-16" #494 prio=5 os_prio=0 tid=0x00007f5d280ee000 nid=0xf4cb6 runnable [0x00007f5cd86f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000421845de0> (a sun.nio.ch.Util$2) - locked <0x0000000421845dd0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000421845cd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-14" #435 prio=5 os_prio=0 tid=0x00007f5cd008b000 nid=0xe12b7 runnable [0x00007f5b2c2e9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004247882c0> (a sun.nio.ch.Util$2) - locked <0x000000042184c2c0> (a java.util.Collections$UnmodifiableSet) - locked <0x000000042184c278> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-14" #434 prio=5 os_prio=0 tid=0x00007f5cd00a4800 nid=0xe12b6 runnable [0x00007f5b2d2f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000041a0a6000> (a sun.nio.ch.Util$2) - locked <0x000000041a0a5ff0> (a java.util.Collections$UnmodifiableSet) - locked <0x000000041a0a5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-13" #428 prio=5 os_prio=0 tid=0x00007f5d140c7800 nid=0xd986c runnable [0x00007f5b2c7ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000041ca2e248> (a sun.nio.ch.Util$2) - locked <0x000000041ca2e238> (a java.util.Collections$UnmodifiableSet) - locked <0x000000041ca2e1f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-13" #427 prio=5 os_prio=0 tid=0x00007f5d140ae000 nid=0xd986b runnable [0x00007f5aedaf8000] java.lang.Thread.State: RUNNABLE at org.apache.log4j.Category.getEffectiveLevel(Category.java:442) at org.apache.log4j.Logger.isTraceEnabled(Logger.java:209) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:137) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:159) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:141) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:141) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:138) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:136) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-11" #374 prio=5 os_prio=0 tid=0x00007f5d080f5800 nid=0x112b5 runnable [0x00007f5b2c6ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000407123518> (a sun.nio.ch.Util$2) - locked <0x0000000407123508> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004071234c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-11" #373 prio=5 os_prio=0 tid=0x00007f5d0807e000 nid=0x112b4 runnable [0x00007f5b2c3ea000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040715d930> (a sun.nio.ch.Util$2) - locked <0x000000040715d920> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040715d8d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-10" #370 prio=5 os_prio=0 tid=0x00007f5ce40c0800 nid=0x8c3c runnable [0x00007f5b2d4fb000] java.lang.Thread.State: RUNNABLE at scala.Tuple2.equals(Tuple2.scala:20) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:940) at kafka.utils.Pool.get(Pool.scala:69) at kafka.server.ReplicaManager.getPartition(ReplicaManager.scala:286) at kafka.server.ReplicaManager.getReplica(ReplicaManager.scala:317) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:126) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:159) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:141) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:141) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:138) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:136) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-10" #369 prio=5 os_prio=0 tid=0x00007f5ce40a6800 nid=0x8c3b runnable [0x00007f5aedcfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004021808a0> (a sun.nio.ch.Util$2) - locked <0x0000000402180890> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000402180848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-9" #365 prio=5 os_prio=0 tid=0x00007f5d0807d000 nid=0x1bbb runnable [0x00007f5cd81ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e2054c58> (a sun.nio.ch.Util$2) - locked <0x00000003e2054c48> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e2054c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-9" #364 prio=5 os_prio=0 tid=0x00007f5d0807c800 nid=0x1bba runnable [0x00007f5cdb6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003e20a9810> (a sun.nio.ch.Util$2) - locked <0x00000003e20a9800> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003e20a97b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-5" #176 prio=5 os_prio=0 tid=0x00007f5d200af000 nid=0xae61b runnable [0x00007f5cd82ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003f42601d8> (a sun.nio.ch.Util$2) - locked <0x00000003f42601c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003f4260180> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-5" #175 prio=5 os_prio=0 tid=0x00007f5d20060000 nid=0xae61a runnable [0x00007f5b2c1e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003f4260510> (a sun.nio.ch.Util$2) - locked <0x00000003f4260500> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003f42604b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-4" #165 prio=5 os_prio=0 tid=0x00007f5ce8097000 nid=0xa5b4c runnable [0x00007f5b2cdf4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000042b67cb40> (a sun.nio.ch.Util$2) - locked <0x000000042b67cb30> (a java.util.Collections$UnmodifiableSet) - locked <0x000000042b67cae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-4" #164 prio=5 os_prio=0 tid=0x00007f5ce807d000 nid=0xa5b4a runnable [0x00007f5b2cff6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004f13d62f8> (a sun.nio.ch.Util$2) - locked <0x00000004f13d62e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004f13d62a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "executor-Produce" #157 prio=5 os_prio=0 tid=0x00007f5d58060000 nid=0x9eaf1 waiting for monitor entry [0x00007f5b2c4eb000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedOperation.run(DelayedOperation.scala:100) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "executor-Fetch" #152 prio=5 os_prio=0 tid=0x00007f5d4c061800 nid=0x9adb7 waiting on condition [0x00007f68e0d50000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee91f480> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ReplicaFetcherThread-0-1" #129 prio=5 os_prio=0 tid=0x00007f5cf06b1000 nid=0x968ca runnable [0x00007f5b2cbf2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004064633e8> (a sun.nio.ch.Util$2) - locked <0x00000004064633d8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000406463390> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-1" #117 prio=5 os_prio=0 tid=0x00007f5cf046e000 nid=0x968be runnable [0x00007f5b2d7fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004066d9028> (a sun.nio.ch.Util$2) - locked <0x00000004066d9018> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004066d8b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-1-2" #114 prio=5 os_prio=0 tid=0x00007f5cf043e000 nid=0x968bb runnable [0x00007f5cd83ee000] java.lang.Thread.State: RUNNABLE at scala.Tuple2.equals(Tuple2.scala:20) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:940) at kafka.utils.Pool.get(Pool.scala:69) at kafka.server.ReplicaManager.getPartition(ReplicaManager.scala:286) at kafka.server.ReplicaManager.getReplica(ReplicaManager.scala:317) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:126) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:159) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:141) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:141) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:138) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:138) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:136) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ReplicaFetcherThread-0-2" #109 prio=5 os_prio=0 tid=0x00007f5cf03f3800 nid=0x968b6 runnable [0x00007f5cd88f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004064242b0> (a sun.nio.ch.Util$2) - locked <0x00000004064242a0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000406424178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:136) at kafka.utils.NetworkClientBlockingOps$.kafka$utils$NetworkClientBlockingOps$$pollContinuously$extension(NetworkClientBlockingOps.scala:143) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:281) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:266) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-9" #108 daemon prio=5 os_prio=0 tid=0x00007f5d3c05d800 nid=0x96881 waiting on condition [0x00007f5cdb3f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-8" #107 daemon prio=5 os_prio=0 tid=0x00007f5d4805d800 nid=0x96875 waiting on condition [0x00007f5cdb4f5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-7" #106 daemon prio=5 os_prio=0 tid=0x00007f5d4405d800 nid=0x96857 runnable [0x00007f5cdb5f6000] java.lang.Thread.State: RUNNABLE at java.nio.MappedByteBuffer.$$YJP$$force0(Native Method) at java.nio.MappedByteBuffer.force0(MappedByteBuffer.java) at java.nio.MappedByteBuffer.force(MappedByteBuffer.java:203) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:140) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:140) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:234) at kafka.log.AbstractIndex.flush(AbstractIndex.scala:139) at kafka.log.LogSegment$$anonfun$flush$1.apply$mcV$sp(LogSegment.scala:317) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:314) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:314) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) at kafka.log.LogSegment.flush(LogSegment.scala:314) at kafka.log.Log$$anonfun$flush$2.apply(Log.scala:926) at kafka.log.Log$$anonfun$flush$2.apply(Log.scala:925) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.flush(Log.scala:925) at kafka.log.Log.flush(Log.scala:911) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$flushDirtyLogs$3.apply(LogManager.scala:472) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$flushDirtyLogs$3.apply(LogManager.scala:466) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:778) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at kafka.utils.Pool$$anon$1.foreach(Pool.scala:89) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.utils.Pool.foreach(Pool.scala:26) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:777) at kafka.log.LogManager.kafka$log$LogManager$$flushDirtyLogs(LogManager.scala:466) at kafka.log.LogManager$$anonfun$startup$2.apply$mcV$sp(LogManager.scala:203) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:110) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:58) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "RMI Scheduler(0)" #103 daemon prio=5 os_prio=0 tid=0x00007f5cc80b0000 nid=0x96850 waiting on condition [0x00007f5cdb8f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8ab0978> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-23" #101 daemon prio=5 os_prio=0 tid=0x00007f6ab11fe800 nid=0x96845 waiting for monitor entry [0x00007f5cdbcfb000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.getOffsets(GroupMetadataManager.scala:372) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleFetchOffsets(GroupCoordinator.scala:473) at kafka.server.KafkaApis.handleOffsetFetchRequest(KafkaApis.scala:927) at kafka.server.KafkaApis.handle(KafkaApis.scala:87) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-22" #100 daemon prio=5 os_prio=0 tid=0x00007f6ab2418000 nid=0x96844 waiting for monitor entry [0x00007f5cdbdfc000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-21" #99 daemon prio=5 os_prio=0 tid=0x00007f6ab2417000 nid=0x96843 waiting for monitor entry [0x00007f5cdbefd000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-20" #98 daemon prio=5 os_prio=0 tid=0x00007f6ab2416800 nid=0x96842 waiting for monitor entry [0x00007f5cdbffe000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-19" #97 daemon prio=5 os_prio=0 tid=0x00007f6ab2420800 nid=0x96841 waiting for monitor entry [0x00007f5d88189000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x0000000520198840> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-18" #96 daemon prio=5 os_prio=0 tid=0x00007f6ab241f800 nid=0x96840 waiting for monitor entry [0x00007f5d8828a000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x0000000520194970> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-17" #95 daemon prio=5 os_prio=0 tid=0x00007f6ab23c8000 nid=0x9683f waiting for monitor entry [0x00007f5d8838c000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-16" #94 daemon prio=5 os_prio=0 tid=0x00007f6ab23c7000 nid=0x9683e waiting for monitor entry [0x00007f5d8848d000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-15" #93 daemon prio=5 os_prio=0 tid=0x00007f6ab241b000 nid=0x9683d waiting for monitor entry [0x00007f5d8858e000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.getOffsets(GroupMetadataManager.scala:372) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleFetchOffsets(GroupCoordinator.scala:473) at kafka.server.KafkaApis.handleOffsetFetchRequest(KafkaApis.scala:927) at kafka.server.KafkaApis.handle(KafkaApis.scala:87) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-14" #92 daemon prio=5 os_prio=0 tid=0x00007f6ab2414800 nid=0x9683c waiting for monitor entry [0x00007f5d8868f000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-13" #91 daemon prio=5 os_prio=0 tid=0x00007f6ab23cc800 nid=0x9683b waiting for monitor entry [0x00007f5d8878f000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x000000052017a200> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-12" #90 daemon prio=5 os_prio=0 tid=0x00007f6ab23bf800 nid=0x9683a waiting for monitor entry [0x00007f5d88891000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-11" #89 daemon prio=5 os_prio=0 tid=0x00007f6ab23f4800 nid=0x96839 waiting for monitor entry [0x00007f5d88992000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-10" #88 daemon prio=5 os_prio=0 tid=0x00007f6ab23b9000 nid=0x96838 waiting for monitor entry [0x00007f5d88a93000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:432) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:420) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:323) at kafka.server.KafkaApis.handle(KafkaApis.scala:86) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-9" #87 daemon prio=5 os_prio=0 tid=0x00007f6ab23f6000 nid=0x96837 waiting for monitor entry [0x00007f5d88b94000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.getOffsets(GroupMetadataManager.scala:372) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleFetchOffsets(GroupCoordinator.scala:473) at kafka.server.KafkaApis.handleOffsetFetchRequest(KafkaApis.scala:927) at kafka.server.KafkaApis.handle(KafkaApis.scala:87) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-8" #86 daemon prio=5 os_prio=0 tid=0x00007f6ab2400000 nid=0x96836 waiting for monitor entry [0x00007f5d88c95000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.getOffsets(GroupMetadataManager.scala:372) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleFetchOffsets(GroupCoordinator.scala:473) at kafka.server.KafkaApis.handleOffsetFetchRequest(KafkaApis.scala:927) at kafka.server.KafkaApis.handle(KafkaApis.scala:87) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-7" #85 daemon prio=5 os_prio=0 tid=0x00007f6ab242f800 nid=0x96835 waiting for monitor entry [0x00007f5d88d95000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x0000000520199530> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-6" #84 daemon prio=5 os_prio=0 tid=0x00007f6ab2429000 nid=0x96834 waiting for monitor entry [0x00007f5d88e96000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x00000005201981d0> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-5" #83 daemon prio=5 os_prio=0 tid=0x00007f6ab23d0800 nid=0x96833 waiting for monitor entry [0x00007f5d88f97000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x0000000520198ec0> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-4" #82 daemon prio=5 os_prio=0 tid=0x00007f6ab11fd800 nid=0x96832 waiting for monitor entry [0x00007f5d89098000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x000000052019bf30> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-3" #81 daemon prio=5 os_prio=0 tid=0x00007f6ab1223800 nid=0x96831 waiting for monitor entry [0x00007f5d89199000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x0000000520199ba0> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-2" #80 daemon prio=5 os_prio=0 tid=0x00007f6ab23d2800 nid=0x96830 waiting for monitor entry [0x00007f5d8929a000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x000000052017b148> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-1" #79 daemon prio=5 os_prio=0 tid=0x00007f6ab23c9800 nid=0x9682f waiting for monitor entry [0x00007f5d8939b000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x0000000520194fe0> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-0" #78 daemon prio=5 os_prio=0 tid=0x00007f6ab1212000 nid=0x9682e waiting for monitor entry [0x00007f5d8949d000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.GroupMetadataManager.getOffsets(GroupMetadataManager.scala:372) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupCoordinator.handleFetchOffsets(GroupCoordinator.scala:473) at kafka.server.KafkaApis.handleOffsetFetchRequest(KafkaApis.scala:927) at kafka.server.KafkaApis.handle(KafkaApis.scala:87) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) "group-metadata-manager-0" #77 daemon prio=5 os_prio=0 tid=0x00007f6ab23b7800 nid=0x9682d waiting for monitor entry [0x00007f5d8959d000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - waiting to lock <0x000000052017a200> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.cluster.Partition.tryCompleteDelayedRequests(Partition.scala:374) at kafka.cluster.Partition.appendMessagesToLeader(Partition.scala:457) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1$$anonfun$apply$10.apply(GroupMetadataManager.scala:600) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1$$anonfun$apply$10.apply(GroupMetadataManager.scala:593) at scala.Option.foreach(Option.scala:257) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1.apply(GroupMetadataManager.scala:593) - locked <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1.apply(GroupMetadataManager.scala:579) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at kafka.utils.Pool$$anon$1.foreach(Pool.scala:89) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.utils.Pool.foreach(Pool.scala:26) at kafka.coordinator.GroupMetadataManager.cleanupGroupMetadata(GroupMetadataManager.scala:579) at kafka.coordinator.GroupMetadataManager$$anonfun$1.apply$mcV$sp(GroupMetadataManager.scala:101) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:110) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:58) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ExpirationReaper-3" #76 prio=5 os_prio=0 tid=0x00007f6ab237e000 nid=0x9682c runnable [0x00007f5d8969f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee346290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:106) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:339) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:363) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-3" #75 prio=5 os_prio=0 tid=0x00007f6ab2371800 nid=0x9682b runnable [0x00007f5d897a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee3492f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:106) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:339) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:363) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-3" #74 prio=5 os_prio=0 tid=0x00007f6ab235c000 nid=0x9682a runnable [0x00007f5d898a1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee348a18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:106) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:339) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:363) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-6" #73 daemon prio=5 os_prio=0 tid=0x00007f68e4049000 nid=0x96829 waiting on condition [0x00007f5d899a2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-5" #72 daemon prio=5 os_prio=0 tid=0x00007f68e805c000 nid=0x96828 waiting on condition [0x00007f5d89aa3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-4" #71 daemon prio=5 os_prio=0 tid=0x00007f6ab22b6000 nid=0x96827 waiting on condition [0x00007f5d89ba4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-3" #70 daemon prio=5 os_prio=0 tid=0x00007f6ab22ed000 nid=0x96826 waiting on condition [0x00007f5d89ca5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-2" #69 daemon prio=5 os_prio=0 tid=0x00007f6ab149c000 nid=0x96825 waiting on condition [0x00007f5d89da6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee349528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ExpirationReaper-3" #68 prio=5 os_prio=0 tid=0x00007f6ab22dd800 nid=0x96824 runnable [0x00007f5d89ea7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee3464e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:106) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:339) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:363) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-3" #67 prio=5 os_prio=0 tid=0x00007f6ab22b9800 nid=0x96823 runnable [0x00007f5d89fa8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee34ab68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:106) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:339) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:363) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-socket-acceptor-PLAINTEXT-9092" #66 prio=5 os_prio=0 tid=0x00007f6ab1a30000 nid=0x96822 runnable [0x00007f5d8a2a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003ee347730> (a sun.nio.ch.Util$2) - locked <0x00000003ee347720> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003ee347740> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:260) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-11" #65 prio=5 os_prio=0 tid=0x00007f6ab1a2f800 nid=0x96821 waiting on condition [0x00007f5d8a3aa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-10" #64 prio=5 os_prio=0 tid=0x00007f6ab1a2e800 nid=0x96820 waiting on condition [0x00007f5d8a4ab000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-9" #63 prio=5 os_prio=0 tid=0x00007f6ab1a2e000 nid=0x9681f waiting on condition [0x00007f5d8a5ac000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-8" #62 prio=5 os_prio=0 tid=0x00007f6ab1a2d000 nid=0x9681e waiting on condition [0x00007f5d8a6ad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-7" #61 prio=5 os_prio=0 tid=0x00007f6ab1a2c800 nid=0x9681d waiting on condition [0x00007f5d8a7ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-6" #60 prio=5 os_prio=0 tid=0x00007f6ab1a2b800 nid=0x9681c waiting on condition [0x00007f5d8a8af000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-5" #59 prio=5 os_prio=0 tid=0x00007f6ab1a2b000 nid=0x9681b waiting on condition [0x00007f5d8a9b0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-4" #58 prio=5 os_prio=0 tid=0x00007f6ab20ce800 nid=0x9681a waiting on condition [0x00007f5d8aab1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-3" #57 prio=5 os_prio=0 tid=0x00007f6ab20d5000 nid=0x96819 waiting on condition [0x00007f5d8abb2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-2" #56 prio=5 os_prio=0 tid=0x00007f6ab1f12800 nid=0x96818 waiting on condition [0x00007f5d8acb3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-1" #55 prio=5 os_prio=0 tid=0x00007f6ab224f800 nid=0x96817 waiting on condition [0x00007f5d8adb4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-3-PLAINTEXT-0" #54 prio=5 os_prio=0 tid=0x00007f6ab2248800 nid=0x96816 waiting on condition [0x00007f696944b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee7aefc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:353) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:212) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:493) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:487) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:487) at kafka.network.Processor.run(SocketServer.scala:417) at java.lang.Thread.run(Thread.java:745) "kafka-log-cleaner-thread-0" #50 prio=5 os_prio=0 tid=0x00007f6ab1eb7000 nid=0x96814 waiting on condition [0x00007f6968c47000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee34b7b8> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:258) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:218) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-2" #53 daemon prio=5 os_prio=0 tid=0x00007f6ab1ea6800 nid=0x96813 waiting on condition [0x00007f6968a45000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-1" #52 daemon prio=5 os_prio=0 tid=0x00007f6ab1eac800 nid=0x96812 waiting on condition [0x00007f696934a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-0" #51 daemon prio=5 os_prio=0 tid=0x00007f6ab1ea5000 nid=0x96811 waiting on condition [0x00007f6968b46000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8352ea0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-1" #49 daemon prio=5 os_prio=0 tid=0x00007f6ab1262800 nid=0x9680d runnable [0x00007f6968944000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003ee349528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "main-EventThread" #42 daemon prio=5 os_prio=0 tid=0x00007f6ab1c19000 nid=0x966e6 waiting on condition [0x00007f6968f48000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d800c568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) "main-SendThread(zk01f.stat.yandex.net:2181)" #41 daemon prio=5 os_prio=0 tid=0x00007f6ab1c12800 nid=0x966e5 runnable [0x00007f6969049000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.$$YJP$$epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.epollWait(EPollArrayWrapper.java) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000003d800c830> (a sun.nio.ch.Util$2) - locked <0x00000003d800c820> (a java.util.Collections$UnmodifiableSet) - locked <0x00000003d800c7d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) "ZkClient-EventThread-40-zk01f.stat.yandex.net:2181,zk02f.stat.yandex.net:2181,zk03f.stat.yandex.net:2181/kafka-myt" #40 daemon prio=5 os_prio=0 tid=0x00007f6ab1b73000 nid=0x966e4 waiting on condition [0x00007f6969970000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8033c08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:67) "ThrottledRequestReaper-Produce" #36 prio=5 os_prio=0 tid=0x00007f6ab1b17800 nid=0x966dd waiting on condition [0x00007f6969a71000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8033e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:158) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ThrottledRequestReaper-Fetch" #35 prio=5 os_prio=0 tid=0x00007f6ab1b2c000 nid=0x966dc waiting on condition [0x00007f6969b72000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d800f548> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:158) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "SensorExpiryThread" #34 daemon prio=5 os_prio=0 tid=0x00007f6ab19e3000 nid=0x966db waiting on condition [0x00007f6969c73000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003d8034010> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-0" #31 daemon prio=5 os_prio=0 tid=0x00007f6ab1002800 nid=0x966d5 runnable [0x00007f696a475000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.$$YJP$$socketAccept(Native Method) at java.net.PlainSocketImpl.socketAccept(PlainSocketImpl.java) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-9999" #30 daemon prio=5 os_prio=0 tid=0x00007f6ab1028000 nid=0x966d4 runnable [0x00007f696a776000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.$$YJP$$socketAccept(Native Method) at java.net.PlainSocketImpl.socketAccept(PlainSocketImpl.java) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-0" #29 daemon prio=5 os_prio=0 tid=0x00007f6ab0f8c000 nid=0x966d3 runnable [0x00007f696a877000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.$$YJP$$socketAccept(Native Method) at java.net.PlainSocketImpl.socketAccept(PlainSocketImpl.java) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "Service Thread" #27 daemon prio=9 os_prio=0 tid=0x00007f6ab0c64800 nid=0x966d1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread14" #26 daemon prio=9 os_prio=0 tid=0x00007f6ab0c10000 nid=0x966cf waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread13" #25 daemon prio=9 os_prio=0 tid=0x00007f6ab0c0e000 nid=0x966ce waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread12" #24 daemon prio=9 os_prio=0 tid=0x00007f6ab0c0c000 nid=0x966cd waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread11" #23 daemon prio=9 os_prio=0 tid=0x00007f6ab0c0a800 nid=0x966cc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread10" #22 daemon prio=9 os_prio=0 tid=0x00007f6ab0c08800 nid=0x966cb waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread9" #21 daemon prio=9 os_prio=0 tid=0x00007f6ab0c06000 nid=0x966ca waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread8" #20 daemon prio=9 os_prio=0 tid=0x00007f6ab0c04000 nid=0x966c9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread7" #19 daemon prio=9 os_prio=0 tid=0x00007f6ab0c02000 nid=0x966c8 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread6" #18 daemon prio=9 os_prio=0 tid=0x00007f6ab0c00000 nid=0x966c7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread5" #17 daemon prio=9 os_prio=0 tid=0x00007f6ab0bf6000 nid=0x966c6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread4" #16 daemon prio=9 os_prio=0 tid=0x00007f6ab0bf3800 nid=0x966c5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread3" #15 daemon prio=9 os_prio=0 tid=0x00007f6ab0be9800 nid=0x966c4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread2" #14 daemon prio=9 os_prio=0 tid=0x00007f6ab0bdf800 nid=0x966c3 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" #13 daemon prio=9 os_prio=0 tid=0x00007f6ab0bd5000 nid=0x966c2 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #12 daemon prio=9 os_prio=0 tid=0x00007f6ab0bd4000 nid=0x966c1 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" #11 daemon prio=10 os_prio=0 tid=0x00007f6ab0bd3800 nid=0x966c0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" #10 daemon prio=10 os_prio=0 tid=0x00007f6ab0bd3000 nid=0x966bf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "YJPAgent-OOMESnapshotDetector" #9 daemon prio=10 os_prio=0 tid=0x00007f6ab0bd0000 nid=0x966be runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "YJPAgent-CPUSampler" #8 daemon prio=10 os_prio=0 tid=0x00007f6ab0bcf000 nid=0x966bd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "YJPAgent-RequestListener" #7 daemon prio=1 os_prio=0 tid=0x00007f6ab0bce800 nid=0x966bc runnable [0x00007f6990466000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.$$YJP$$socketAccept(Native Method) at java.net.PlainSocketImpl.socketAccept(PlainSocketImpl.java) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at com.yourkit.runtime.Core$4.run(Core.java:726) at java.lang.Thread.run(Thread.java:745) "YJPAgent-Telemetry" #6 daemon prio=5 os_prio=0 tid=0x00007f6ab0bcd800 nid=0x966bb waiting on condition [0x00007f6990567000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.$$YJP$$sleep(Native Method) at java.lang.Thread.sleep(Thread.java) at com.yourkit.util.Util.sleep(Util.java:64) at com.yourkit.runtime.TelemetryThread.run(TelemetryThread.java:579) "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f6ab0b00800 nid=0x966b8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f6ab0aaa000 nid=0x966b7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f6ab095a000 nid=0x966b6 in Object.wait() [0x00007f6992988000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.$$YJP$$wait(Native Method) at java.lang.Object.wait(Object.java) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000003d800fa70> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f6ab09b5000 nid=0x966b5 in Object.wait() [0x00007f6992a89000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.$$YJP$$wait(Native Method) at java.lang.Object.wait(Object.java) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00000003d8044b50> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) "main" #1 prio=5 os_prio=0 tid=0x00007f6ab014c800 nid=0x9667d waiting on condition [0x00007f6ab886d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000003efaae830> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:651) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:70) at kafka.Kafka$.main(Kafka.scala:68) at kafka.Kafka.main(Kafka.scala) "VM Thread" os_prio=0 tid=0x00007f6ab0982000 nid=0x966b4 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01be000 nid=0x9667e runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01da000 nid=0x9667f runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01db800 nid=0x96680 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01dd000 nid=0x96681 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01df000 nid=0x96682 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01e0800 nid=0x96683 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01e2800 nid=0x96684 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01e4000 nid=0x96685 runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01e6000 nid=0x96686 runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01e7800 nid=0x96687 runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01e9800 nid=0x96688 runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01eb000 nid=0x96689 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01ed000 nid=0x9668a runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01ee800 nid=0x9668b runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01f0800 nid=0x9668c runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01f2000 nid=0x9668d runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01f4000 nid=0x9668e runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01f5800 nid=0x9668f runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01f7800 nid=0x96690 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01f9000 nid=0x96691 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01fb000 nid=0x96692 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01fc800 nid=0x96693 runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f6ab01fe800 nid=0x96694 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f6ab02bc000 nid=0x966ad runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f6ab02be000 nid=0x966ae runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f6ab02bf800 nid=0x966af runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f6ab02c1800 nid=0x966b0 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f6ab02c3000 nid=0x966b1 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f6ab02c5000 nid=0x966b2 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f6ab02c6800 nid=0x966b3 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f6ab022b000 nid=0x966ac runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f6ab0229000 nid=0x966ab runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f6ab0227800 nid=0x966aa runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f6ab0225800 nid=0x966a9 runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f6ab0223800 nid=0x966a8 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f6ab0222000 nid=0x966a7 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f6ab0220000 nid=0x966a6 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f6ab021e800 nid=0x966a5 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f6ab021c800 nid=0x966a4 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f6ab021b000 nid=0x966a3 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f6ab0219000 nid=0x966a2 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f6ab0217800 nid=0x966a1 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f6ab0215800 nid=0x966a0 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f6ab0213800 nid=0x9669f runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f6ab0212000 nid=0x9669e runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f6ab0210000 nid=0x9669d runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f6ab020e800 nid=0x9669c runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f6ab020c800 nid=0x9669b runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f6ab020b000 nid=0x9669a runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f6ab0209000 nid=0x96699 runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f6ab0207800 nid=0x96698 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f6ab0205800 nid=0x96697 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f6ab0203800 nid=0x96696 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f6ab0202000 nid=0x96695 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f6ab1048800 nid=0x966d6 waiting on condition JNI global references: 911445 Found one Java-level deadlock: ============================= "executor-Produce": waiting to lock monitor 0x00007f5d5c20f928 (object 0x0000000433a1d1d0, a kafka.coordinator.GroupMetadata), which is held by "group-metadata-manager-0" "group-metadata-manager-0": waiting to lock monitor 0x00007f5d7d686558 (object 0x000000052017a200, a kafka.server.DelayedProduce), which is held by "kafka-request-handler-13" "kafka-request-handler-13": waiting to lock monitor 0x00007f5d5c20f928 (object 0x0000000433a1d1d0, a kafka.coordinator.GroupMetadata), which is held by "group-metadata-manager-0" Java stack information for the threads listed above: =================================================== "executor-Produce": at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedOperation.run(DelayedOperation.scala:100) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "group-metadata-manager-0": at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - waiting to lock <0x000000052017a200> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.cluster.Partition.tryCompleteDelayedRequests(Partition.scala:374) at kafka.cluster.Partition.appendMessagesToLeader(Partition.scala:457) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1$$anonfun$apply$10.apply(GroupMetadataManager.scala:600) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1$$anonfun$apply$10.apply(GroupMetadataManager.scala:593) at scala.Option.foreach(Option.scala:257) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1.apply(GroupMetadataManager.scala:593) - locked <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$cleanupGroupMetadata$1.apply(GroupMetadataManager.scala:579) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at kafka.utils.Pool$$anon$1.foreach(Pool.scala:89) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at kafka.utils.Pool.foreach(Pool.scala:26) at kafka.coordinator.GroupMetadataManager.cleanupGroupMetadata(GroupMetadataManager.scala:579) at kafka.coordinator.GroupMetadataManager$$anonfun$1.apply$mcV$sp(GroupMetadataManager.scala:101) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:110) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:58) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-13": at kafka.coordinator.GroupMetadataManager.kafka$coordinator$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:301) - waiting to lock <0x0000000433a1d1d0> (a kafka.coordinator.GroupMetadata) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.coordinator.GroupMetadataManager$$anonfun$prepareStoreOffsets$1.apply(GroupMetadataManager.scala:357) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:123) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:64) at kafka.server.DelayedProduce.tryComplete(DelayedProduce.scala:105) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:312) - locked <0x000000052017a200> (a kafka.server.DelayedProduce) at kafka.server.DelayedOperationPurgatory$Watchers$$anonfun$tryCompleteWatched$1.apply(DelayedOperation.scala:309) at scala.collection.Iterator$class.foreach(Iterator.scala:742) at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:309) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:228) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:200) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:915) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:908) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:908) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:481) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:524) at kafka.server.KafkaApis.handle(KafkaApis.scala:79) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:60) at java.lang.Thread.run(Thread.java:745) Found 1 deadlock.